#include <map>
#include <set>
#include <cmath>
#include <string>
#include <vector>
#include <fstream>
#include <sstream>
#include <iostream>

using namespace std;

const int DOC_COUNT = 4043523;
map<string, int> appearDocCount;
// word, docid, tf*idf
map<string, vector<pair<int, double> > > index;

void gao(const string& buffer, int id)
{
    stringstream ss(buffer);
    string word;
    set<string> hasAppear;
    int currentDocWordCount = 0;
    map<string, int> currentAppearCount;

    while (ss >> word) {
        ++currentDocWordCount;
        if (appearDocCount.find(word) == appearDocCount.end()) {
            continue;
        }
        if (hasAppear.find(word) == hasAppear.end()) {
            ++appearDocCount[word];
            hasAppear.insert(word);
        }
        ++currentAppearCount[word];
    }
    if (currentDocWordCount == 0) {
        return;
    }
    for (set<string>::iterator p = hasAppear.begin(); p != hasAppear.end(); ++p) {
        index[*p].push_back(make_pair(id, 1.0 * currentAppearCount[*p] / currentDocWordCount));
    }
}

int main(int argc, const char *argv[])
{
    string word;
    string buffer;
    ifstream fin1("dict");
    ifstream fin2("../wiki_data/wiki_text_filtered_stemming");   
    ofstream fout("index");

    while (fin1 >> word) {
        appearDocCount[word] = 0;
    }
    
    int id = 0;
    while (getline(fin2, buffer)) {
        gao(buffer, id);
        ++id;
    }

    for (map<string, vector<pair<int, double> > >::iterator p = index.begin(); p != index.end(); ++p) {
        vector<pair<int, double> >& tmp = p->second;
        fout << p->first << endl;
        double idf = log10(1.0 * DOC_COUNT / appearDocCount[p->first]);
        for (int i = 0; i < tmp.size(); ++i) {
            tmp[i].second *= idf;
            fout << tmp[i].first << ' ' << tmp[i].second << ' ';
        }
        fout << endl;
    }
    
    return 0;
}
